<!doctype html>














<!-- `site.alt_lang` can specify a language different from the UI -->
<html lang="en" 
  
>
  <!-- The Head -->

<head>
  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  <meta name="theme-color" media="(prefers-color-scheme: light)" content="#f7f7f7">
  <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#1b1b1e">
  <meta name="apple-mobile-web-app-capable" content="yes">
  <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
  <meta
    name="viewport"
    content="width=device-width, user-scalable=no initial-scale=1, shrink-to-fit=no, viewport-fit=cover"
  >

  

  

  
    <!-- Begin Jekyll SEO tag v2.8.0 -->
<meta name="generator" content="Jekyll v4.3.2" />
<meta property="og:title" content="Kernel Virtual Function I/O" />
<meta name="author" content="Chunsheng Luo" />
<meta property="og:locale" content="en" />
<meta name="description" content="一、vfio" />
<meta property="og:description" content="一、vfio" />
<link rel="canonical" href="https://luochenglcs.github.io/posts/linux-vfio/" />
<meta property="og:url" content="https://luochenglcs.github.io/posts/linux-vfio/" />
<meta property="og:site_name" content="Cheng Luo" />
<meta property="og:type" content="article" />
<meta property="article:published_time" content="2022-12-13T00:00:00+00:00" />
<meta name="twitter:card" content="summary" />
<meta property="twitter:title" content="Kernel Virtual Function I/O" />
<meta name="twitter:site" content="@luochenglcs" />
<meta name="twitter:creator" content="@Chunsheng Luo" />
<script type="application/ld+json">
{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Chunsheng Luo","url":"https://github.com/luochenglcs"},"dateModified":"2023-08-07T10:25:19+00:00","datePublished":"2022-12-13T00:00:00+00:00","description":"一、vfio","headline":"Kernel Virtual Function I/O","mainEntityOfPage":{"@type":"WebPage","@id":"https://luochenglcs.github.io/posts/linux-vfio/"},"url":"https://luochenglcs.github.io/posts/linux-vfio/"}</script>
<!-- End Jekyll SEO tag -->

  

  <title>Kernel Virtual Function I/O | Cheng Luo
  </title>

  <!--
  The Favicons for Web, Android, Microsoft, and iOS (iPhone and iPad) Apps
  Generated by: https://realfavicongenerator.net/
-->



<link rel="apple-touch-icon" sizes="180x180" href="/assets/img/favicons/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/assets/img/favicons/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/assets/img/favicons/favicon-16x16.png">
<link rel="manifest" href="/assets/img/favicons/site.webmanifest">
<link rel="shortcut icon" href="/assets/img/favicons/favicon.ico">
<meta name="apple-mobile-web-app-title" content="Cheng Luo">
<meta name="application-name" content="Cheng Luo">
<meta name="msapplication-TileColor" content="#da532c">
<meta name="msapplication-config" content="/assets/img/favicons/browserconfig.xml">
<meta name="theme-color" content="#ffffff">


  
    
      <link rel="preconnect" href="https://fonts.googleapis.com" >
      <link rel="dns-prefetch" href="https://fonts.googleapis.com" >
    
      <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
      <link rel="dns-prefetch" href="https://fonts.gstatic.com" crossorigin>
    
      <link rel="preconnect" href="https://fonts.googleapis.com" >
      <link rel="dns-prefetch" href="https://fonts.googleapis.com" >
    
      <link rel="preconnect" href="https://cdn.jsdelivr.net" >
      <link rel="dns-prefetch" href="https://cdn.jsdelivr.net" >
    

    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Lato&family=Source+Sans+Pro:wght@400;600;700;900&display=swap">
  

  <!-- GA -->
  

  <!-- Bootstrap -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/css/bootstrap.min.css">

  <!-- Font Awesome -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@6.4.0/css/all.min.css">

  <link rel="stylesheet" href="/assets/css/style.css">

  
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/tocbot@4.21.0/dist/tocbot.min.css">
  

  
    <!-- Manific Popup -->
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/magnific-popup@1.1.0/dist/magnific-popup.min.css">
  

  <!-- JavaScript -->

  
    <!-- Switch the mode between dark and light. -->

<script type="text/javascript">
  class ModeToggle {
    static get MODE_KEY() {
      return 'mode';
    }
    static get MODE_ATTR() {
      return 'data-mode';
    }
    static get DARK_MODE() {
      return 'dark';
    }
    static get LIGHT_MODE() {
      return 'light';
    }
    static get ID() {
      return 'mode-toggle';
    }

    constructor() {
      if (this.hasMode) {
        if (this.isDarkMode) {
          if (!this.isSysDarkPrefer) {
            this.setDark();
          }
        } else {
          if (this.isSysDarkPrefer) {
            this.setLight();
          }
        }
      }

      let self = this;

      /* always follow the system prefers */
      this.sysDarkPrefers.addEventListener('change', () => {
        if (self.hasMode) {
          if (self.isDarkMode) {
            if (!self.isSysDarkPrefer) {
              self.setDark();
            }
          } else {
            if (self.isSysDarkPrefer) {
              self.setLight();
            }
          }

          self.clearMode();
        }

        self.notify();
      });
    } /* constructor() */

    get sysDarkPrefers() {
      return window.matchMedia('(prefers-color-scheme: dark)');
    }

    get isSysDarkPrefer() {
      return this.sysDarkPrefers.matches;
    }

    get isDarkMode() {
      return this.mode === ModeToggle.DARK_MODE;
    }

    get isLightMode() {
      return this.mode === ModeToggle.LIGHT_MODE;
    }

    get hasMode() {
      return this.mode != null;
    }

    get mode() {
      return sessionStorage.getItem(ModeToggle.MODE_KEY);
    }

    /* get the current mode on screen */
    get modeStatus() {
      if (this.isDarkMode || (!this.hasMode && this.isSysDarkPrefer)) {
        return ModeToggle.DARK_MODE;
      } else {
        return ModeToggle.LIGHT_MODE;
      }
    }

    setDark() {
      document.documentElement.setAttribute(ModeToggle.MODE_ATTR, ModeToggle.DARK_MODE);
      sessionStorage.setItem(ModeToggle.MODE_KEY, ModeToggle.DARK_MODE);
    }

    setLight() {
      document.documentElement.setAttribute(ModeToggle.MODE_ATTR, ModeToggle.LIGHT_MODE);
      sessionStorage.setItem(ModeToggle.MODE_KEY, ModeToggle.LIGHT_MODE);
    }

    clearMode() {
      document.documentElement.removeAttribute(ModeToggle.MODE_ATTR);
      sessionStorage.removeItem(ModeToggle.MODE_KEY);
    }

    /* Notify another plugins that the theme mode has changed */
    notify() {
      window.postMessage(
        {
          direction: ModeToggle.ID,
          message: this.modeStatus
        },
        '*'
      );
    }

    flipMode() {
      if (this.hasMode) {
        if (this.isSysDarkPrefer) {
          if (this.isLightMode) {
            this.clearMode();
          } else {
            this.setLight();
          }
        } else {
          if (this.isDarkMode) {
            this.clearMode();
          } else {
            this.setDark();
          }
        }
      } else {
        if (this.isSysDarkPrefer) {
          this.setLight();
        } else {
          this.setDark();
        }
      }

      this.notify();
    } /* flipMode() */
  } /* ModeToggle */

  const modeToggle = new ModeToggle();
</script>

  

  <!-- A placeholder to allow defining custom metadata -->

</head>


  <body>
    <!-- The Side Bar -->

<div id="sidebar" class="d-flex flex-column align-items-end">
  <div class="profile-wrapper">
    <a href="/" id="avatar" class="rounded-circle">
      
        
        <img src="/assets/img/favicons/android-chrome-512x512.png" width="112" height="112" alt="avatar" onerror="this.style.display='none'">
      
    </a>

    <div class="site-title">
      <a href="/">Cheng Luo</a>
    </div>
    <div class="site-subtitle fst-italic">Linux operating system engineer</div>
  </div>
  <!-- .profile-wrapper -->

  <ul class="nav flex-column flex-grow-1 w-100 ps-0">
    <!-- home -->
    <li class="nav-item">
      <a href="/" class="nav-link">
        <i class="fa-fw fas fa-home"></i>
        <span>HOME</span>
      </a>
    </li>
    <!-- the real tabs -->
    
      <li class="nav-item">
        <a href="/categories/" class="nav-link">
          <i class="fa-fw fas fa-stream"></i>
          

          <span>CATEGORIES</span>
        </a>
      </li>
      <!-- .nav-item -->
    
      <li class="nav-item">
        <a href="/tags/" class="nav-link">
          <i class="fa-fw fas fa-tags"></i>
          

          <span>TAGS</span>
        </a>
      </li>
      <!-- .nav-item -->
    
      <li class="nav-item">
        <a href="/archives/" class="nav-link">
          <i class="fa-fw fas fa-archive"></i>
          

          <span>ARCHIVES</span>
        </a>
      </li>
      <!-- .nav-item -->
    
      <li class="nav-item">
        <a href="/about/" class="nav-link">
          <i class="fa-fw fas fa-info-circle"></i>
          

          <span>ABOUT</span>
        </a>
      </li>
      <!-- .nav-item -->
    
  </ul>
  <!-- ul.nav.flex-column -->

  <div class="sidebar-bottom d-flex flex-wrap  align-items-center w-100">
    
      <button class="mode-toggle btn" aria-label="Switch Mode">
        <i class="fas fa-adjust"></i>
      </button>

      
        <span class="icon-border"></span>
      
    

    
      

      
        <a
          href="https://github.com/luochenglcs"
          aria-label="github"
          

          
            target="_blank"
            
          

          

          
            rel="noopener noreferrer"
          
        >
          <i class="fab fa-github"></i>
        </a>
      
    
      

      
    
      

      
        <a
          href="javascript:location.href = 'mailto:' + ['luochunsheng','ustc.edu'].join('@')"
          aria-label="email"
          

          

          

          
        >
          <i class="fas fa-envelope"></i>
        </a>
      
    
  </div>
  <!-- .sidebar-bottom -->
</div>
<!-- #sidebar -->


    <div id="main-wrapper" class="d-flex justify-content-center">
      <div id="main" class="container px-xxl-5">
        <!-- The Top Bar -->

<div id="topbar-wrapper">
  <div
    id="topbar"
    class="container d-flex align-items-center justify-content-between h-100"
  >
    <span id="breadcrumb">
      

      
        
          
            <span>
              <a href="/">
                Home
              </a>
            </span>

          
        
          
        
          
            
              <span>Kernel Virtual Function I/O</span>
            

          
        
      
    </span>
    <!-- endof #breadcrumb -->

    <i id="sidebar-trigger" class="fas fa-bars fa-fw"></i>

    <div id="topbar-title">
      Post
    </div>

    <i id="search-trigger" class="fas fa-search fa-fw"></i>
    <span id="search-wrapper" class="align-items-center">
      <i class="fas fa-search fa-fw"></i>
      <input
        class="form-control"
        id="search-input"
        type="search"
        aria-label="search"
        autocomplete="off"
        placeholder="Search..."
      >
    </span>
    <span id="search-cancel">Cancel</span>
  </div>
</div>

        











<div class="row">
  <!-- core -->
  <div id="core-wrapper" class="col-12 col-lg-11 col-xl-9 pe-xl-4">
    

    <div class="post px-1 px-md-2">
      

      
        
      
        <!-- Refactor the HTML structure -->



<!--
  In order to allow a wide table to scroll horizontally,
  we suround the markdown table with `<div class="table-wrapper">` and `</div>`
-->



<!--
  Fixed kramdown code highlight rendering:
  https://github.com/penibelst/jekyll-compress-html/issues/101
  https://github.com/penibelst/jekyll-compress-html/issues/71#issuecomment-188144901
-->



<!-- Change the icon of checkbox -->


<!-- images -->



  
  

  <!-- CDN URL -->
  

  <!-- Add image path -->
  

  
    
      
      
    

    
    

    

    
    

    
    
    
    

    
      

      
      
      

      
    
      

      
      
      

      
    

    <!-- take out classes -->
    

    

    

    

    <!-- lazy-load images <https://github.com/aFarkas/lazysizes#readme> -->
    
    

    <!-- add image placeholder -->
    
      
    

    <!-- Bypass the HTML-proofer test -->
    

    
      <!-- make sure the `<img>` is wrapped by `<a>` -->
      

      
        <!-- create the image wrapper -->
        
      
    

    <!-- combine -->
    

  
    

    
    

    

    
    

    
    
    
    

    
      

      
      
      

      
    
      

      
      
      

      
    

    <!-- take out classes -->
    

    

    

    

    <!-- lazy-load images <https://github.com/aFarkas/lazysizes#readme> -->
    
    

    <!-- add image placeholder -->
    
      
    

    <!-- Bypass the HTML-proofer test -->
    

    
      <!-- make sure the `<img>` is wrapped by `<a>` -->
      

      
        <!-- create the image wrapper -->
        
      
    

    <!-- combine -->
    

  

  



<!-- Add header for code snippets -->



<!-- Create heading anchors -->





  
  

  

  
  

  
    
    

    
      
        
        
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    

    

  

  
  

  
    
    

    
      
        
        
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    

    

  

  
  

  




<!-- return -->




<h1 data-toc-skip>Kernel Virtual Function I/O</h1>

<div class="post-meta text-muted">
    <!-- published date -->
    <span>
      Posted
      <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class=""
  data-ts="1670889600"
  data-df="ll"
  
    data-bs-toggle="tooltip" data-bs-placement="bottom"
  
>
  Dec 13, 2022
</em>

    </span>

    <!-- lastmod date -->
    
    <span>
      Updated
      <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class=""
  data-ts="1691403919"
  data-df="ll"
  
    data-bs-toggle="tooltip" data-bs-placement="bottom"
  
>
  Aug  7, 2023
</em>

    </span>
    

  

  <div class="d-flex justify-content-between">
    <!-- author(s) -->
    <span>
      

      By

      <em>
      
        
          <a href="https://github.com/luochenglcs">Chunsheng Luo</a>
          
        
      
      </em>
    </span>

    <div>
      <!-- read time -->
      <!-- Calculate the post's reading time, and display the word count in tooltip -->



<!-- words per minute -->










<!-- return element -->
<span
  class="readtime"
  data-bs-toggle="tooltip"
  data-bs-placement="bottom"
  title="2923 words"
>
  <em>16 min</em> read</span>

    </div>

  </div> <!-- .d-flex -->

</div> <!-- .post-meta -->
<div class="post-content">
  <h3 id="一vfio"><span class="me-2">一、vfio</span><a href="#一vfio" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h3>

<p>Virtual Function I/O (VFIO) 是一种现代化的设备直通方案，它充分利用了VT-d/AMD-Vi技术提供的DMA Remapping和Interrupt Remapping特性， 在保证直通设备的DMA安全性同时可以达到接近物理设备的I/O的性能。 用户态进程可以直接使用VFIO驱动直接访问硬件，并且由于整个过程是在IOMMU的保护下进行因此十分安全， 而且非特权用户也是可以直接使用。 换句话说，VFIO是一套完整的用户态驱动(userspace driver)方案，因为它可以安全地把设备I/O、中断、DMA等能力呈现给用户空间。</p>

<p>为了达到最高的IO性能，虚拟机就需要VFIO这种设备直通方式，因为它具有低延时、高带宽的特点，并且guest也能够直接使用设备的原生驱动。 这些优异的特点得益于VFIO对VT-d/AMD-Vi所提供的DMA Remapping和Interrupt Remapping机制的应用。 VFIO使用DMA Remapping为每个Domain建立独立的IOMMU Page Table将直通设备的DMA访问限制在Domain的地址空间之内保证了用户态DMA的安全性， 使用Interrupt Remapping来完成中断重映射和Interrupt Posting来达到中断隔离和中断直接投递的目的。</p>

<p><a href="/posts/.\images\linux-vfio\vfio管理概念.png" class="popup img-link "><img data-src="/posts/.\images\linux-vfio\vfio管理概念.png" alt="image-20221208102036619" class="lazyload" data-proofer-ignore></a></p>

<p>在了解VFIO之前需要了解3个基本概念：device, group, container，它们在逻辑上的关系如上图所示。</p>

<ul>
  <li>Group 是IOMMU能够进行DMA隔离的最小硬件单元，一个group内可能只有一个device，也可能有多个device，这取决于物理平台上硬件的IOMMU拓扑结构。 设备直通的时候一个group里面的设备必须都直通给一个虚拟机。 不能够让一个group里的多个device分别从属于2个不同的VM，也不允许部分device在host上而另一部分被分配到guest里， 因为就这样一个guest中的device可以利用DMA攻击获取另外一个guest里的数据，就无法做到物理上的DMA隔离。 另外，VFIO中的group和iommu group可以认为是同一个概念。</li>
  <li>Device 指的是我们要操作的硬件设备，不过这里的“设备”需要从IOMMU拓扑的角度去理解。如果该设备是一个硬件拓扑上独立的设备，那么它自己就构成一个iommu group。 如果这里是一个multi-function设备，那么它和其他的function一起组成一个iommu group，因为多个function设备在物理硬件上就是互联的， 他们可以互相访问对方的数据，所以必须放到一个group里隔离起来。值得一提的是，对于支持PCIe ACS特性的硬件设备，我们可以认为他们在物理上是互相隔离的。</li>
  <li>Container 是一个和地址空间相关联的概念，这里可以简单把它理解为一个VM Domain的物理内存空间。</li>
</ul>

<p>从上图可以看出，一个或多个device从属于某个group，而一个或多个group又从属于一个container。 如果要将一个device直通给VM，那么先要找到这个设备从属的iommu group，然后将整个group加入到container中即可。</p>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
9
10
11
12
13
</pre></td><td class="rouge-code"><pre><span class="o">+-------------------------------------------+</span>
<span class="o">|</span>                                           <span class="o">|</span>
<span class="o">|</span>             <span class="n">VFIO</span> <span class="n">Interface</span>                <span class="o">|</span>
<span class="o">|</span>                                           <span class="o">|</span>
<span class="o">+---------------------+---------------------+</span>
<span class="o">|</span>                     <span class="o">|</span>                     <span class="o">|</span>
<span class="o">|</span>     <span class="n">vfio_iommu</span>      <span class="o">|</span>      <span class="n">vfio_pci</span>       <span class="o">|</span>
<span class="o">|</span>                     <span class="o">|</span>                     <span class="o">|</span>
<span class="o">+---------------------+---------------------+</span>
<span class="o">|</span>                     <span class="o">|</span>                     <span class="o">|</span>
<span class="o">|</span>    <span class="n">iommu</span> <span class="n">driver</span>     <span class="o">|</span>    <span class="n">pci_bus</span> <span class="n">driver</span>   <span class="o">|</span>
<span class="o">|</span>                     <span class="o">|</span>                     <span class="o">|</span>
<span class="o">+---------------------+---------------------+</span>
</pre></td></tr></tbody></table></code></div></div>

<p>例子：</p>

<p>vfio的样例：</p>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
</pre></td><td class="rouge-code"><pre><span class="cp">#include</span> <span class="cpf">&lt;stdio.h&gt;</span><span class="cp">
#include</span> <span class="cpf">&lt;unistd.h&gt;</span><span class="cp">
#include</span> <span class="cpf">&lt;sys/types.h&gt;</span><span class="cp">
#include</span> <span class="cpf">&lt;sys/stat.h&gt;</span><span class="cp">
#include</span> <span class="cpf">&lt;fcntl.h&gt;</span><span class="cp">
#include</span> <span class="cpf">&lt;linux/vfio.h&gt;</span><span class="cp">
#include</span> <span class="cpf">&lt;sys/mman.h&gt;</span><span class="cp">
#include</span> <span class="cpf">&lt;sys/ioctl.h&gt;</span><span class="cp">
#include</span> <span class="cpf">&lt;string.h&gt;</span><span class="cp">
#include</span> <span class="cpf">&lt;stdlib.h&gt;</span><span class="cp">
</span>
<span class="cp">#define VFIO_PATH "/dev/vfio/vfio"
#define DEVICE_ID "0000:ba:00.0" // BDF of the device taken over by vfio.
</span>
<span class="k">static</span> <span class="kt">int</span> <span class="nf">vfio_get_group_num</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span> <span class="o">*</span><span class="n">linkname</span><span class="p">,</span> <span class="kt">int</span> <span class="o">*</span><span class="n">iommu_group_num</span><span class="p">)</span>
<span class="p">{</span>
	<span class="kt">char</span> <span class="n">filename</span><span class="p">[</span><span class="mi">256</span><span class="p">];</span>
	<span class="kt">char</span> <span class="o">*</span><span class="n">group_tok</span><span class="p">,</span> <span class="o">*</span><span class="n">end</span><span class="p">;</span>
	<span class="kt">int</span> <span class="n">ret</span><span class="p">;</span>

	<span class="n">memset</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">filename</span><span class="p">));</span>

	<span class="n">ret</span> <span class="o">=</span> <span class="n">readlink</span><span class="p">(</span><span class="n">linkname</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">filename</span><span class="p">));</span>

	<span class="cm">/* if the link doesn't exist, no VFIO for us */</span>
	<span class="k">if</span> <span class="p">(</span><span class="n">ret</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span>
		<span class="k">return</span> <span class="mi">0</span><span class="p">;</span>

	<span class="n">printf</span><span class="p">(</span><span class="s">"%s</span><span class="se">\n</span><span class="s">"</span><span class="p">,</span> <span class="n">filename</span><span class="p">);</span>

	<span class="n">end</span> <span class="o">=</span> <span class="n">strtok</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s">"/"</span><span class="p">);</span>

	<span class="k">while</span> <span class="p">(</span><span class="n">end</span> <span class="o">!=</span> <span class="nb">NULL</span><span class="p">)</span> <span class="p">{</span>
		<span class="n">group_tok</span> <span class="o">=</span> <span class="n">end</span><span class="p">;</span>
		<span class="n">end</span> <span class="o">=</span> <span class="n">strtok</span><span class="p">(</span><span class="nb">NULL</span><span class="p">,</span> <span class="s">"/"</span><span class="p">);</span>
	<span class="p">}</span>

	<span class="cm">/* IOMMU group is always the last token */</span>
	<span class="o">*</span><span class="n">iommu_group_num</span> <span class="o">=</span> <span class="n">strtol</span><span class="p">(</span><span class="n">group_tok</span><span class="p">,</span> <span class="nb">NULL</span><span class="p">,</span> <span class="mi">10</span><span class="p">);</span>
	<span class="k">return</span> <span class="mi">1</span><span class="p">;</span>
<span class="p">}</span>

<span class="kt">int</span> <span class="nf">main</span><span class="p">()</span>
<span class="p">{</span>
	<span class="kt">int</span> <span class="n">container</span><span class="p">,</span> <span class="n">group</span><span class="p">,</span> <span class="n">device</span><span class="p">,</span> <span class="n">i</span><span class="p">;</span>
	<span class="k">struct</span> <span class="n">vfio_group_status</span> <span class="n">group_status</span> <span class="o">=</span>
	    <span class="p">{.</span><span class="n">argsz</span> <span class="o">=</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">group_status</span><span class="p">)</span> <span class="p">};</span>
	<span class="k">struct</span> <span class="n">vfio_iommu_type1_info</span> <span class="n">iommu_info</span> <span class="o">=</span> <span class="p">{.</span><span class="n">argsz</span> <span class="o">=</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">iommu_info</span><span class="p">)</span>
	<span class="p">};</span>
	<span class="k">struct</span> <span class="n">vfio_iommu_type1_dma_map</span> <span class="n">dma_map</span> <span class="o">=</span> <span class="p">{.</span><span class="n">argsz</span> <span class="o">=</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">dma_map</span><span class="p">)</span> <span class="p">};</span>
	<span class="k">struct</span> <span class="n">vfio_device_info</span> <span class="n">device_info</span> <span class="o">=</span> <span class="p">{.</span><span class="n">argsz</span> <span class="o">=</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">device_info</span><span class="p">)</span> <span class="p">};</span>
	<span class="kt">int</span> <span class="n">ret</span><span class="p">;</span>

	<span class="cm">/* Create a new container */</span>
	<span class="n">container</span> <span class="o">=</span> <span class="n">open</span><span class="p">(</span><span class="s">"/dev/vfio/vfio"</span><span class="p">,</span> <span class="n">O_RDWR</span><span class="p">);</span>
	<span class="k">if</span> <span class="p">(</span><span class="n">container</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error container.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>

	<span class="cm">/* Unknown API version */</span>
	<span class="k">if</span> <span class="p">(</span><span class="n">ioctl</span><span class="p">(</span><span class="n">container</span><span class="p">,</span> <span class="n">VFIO_GET_API_VERSION</span><span class="p">)</span> <span class="o">!=</span> <span class="n">VFIO_API_VERSION</span><span class="p">)</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error: Unknown VFIO API version.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>

	<span class="cm">/* Doesn't support the IOMMU driver we want. */</span>
	<span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="n">ioctl</span><span class="p">(</span><span class="n">container</span><span class="p">,</span> <span class="n">VFIO_CHECK_EXTENSION</span><span class="p">,</span> <span class="n">VFIO_TYPE1_IOMMU</span><span class="p">))</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error: Doesn't support the IOMMU driver we want.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>

	<span class="cm">/* Get the device's group index *sys/bus/pci/devices/0000\:60\:00.0/iommu_group*/</span>
	<span class="kt">char</span> <span class="o">*</span><span class="n">sys_path</span> <span class="o">=</span> <span class="s">"/sys/bus/pci/devices/"</span> <span class="n">DEVICE_ID</span> <span class="s">"/iommu_group"</span><span class="p">;</span>
	<span class="kt">int</span> <span class="n">group_num</span><span class="p">;</span>
	<span class="kt">char</span> <span class="n">group_name</span><span class="p">[</span><span class="mi">20</span><span class="p">];</span>
	<span class="n">memset</span><span class="p">(</span><span class="n">group_name</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">group_name</span><span class="p">));</span>
	<span class="n">vfio_get_group_num</span><span class="p">(</span><span class="n">sys_path</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">group_num</span><span class="p">);</span>
	<span class="n">printf</span><span class="p">(</span><span class="s">"group_num=%d</span><span class="se">\n</span><span class="s">"</span><span class="p">,</span> <span class="n">group_num</span><span class="p">);</span>
	<span class="n">snprintf</span><span class="p">(</span><span class="n">group_name</span><span class="p">,</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">group_name</span><span class="p">),</span> <span class="s">"/dev/vfio/%d"</span><span class="p">,</span> <span class="n">group_num</span><span class="p">);</span>

	<span class="cm">/* Open the group */</span>
	<span class="n">group</span> <span class="o">=</span> <span class="n">open</span><span class="p">(</span><span class="n">group_name</span><span class="p">,</span> <span class="n">O_RDWR</span><span class="p">);</span>

	<span class="cm">/* Test the group is viable and available */</span>
	<span class="n">ioctl</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">VFIO_GROUP_GET_STATUS</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">group_status</span><span class="p">);</span>

	<span class="cm">/* Group is not viable (ie, not all devices bound for vfio) */</span>
	<span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="p">(</span><span class="n">group_status</span><span class="p">.</span><span class="n">flags</span> <span class="o">&amp;</span> <span class="n">VFIO_GROUP_FLAGS_VIABLE</span><span class="p">))</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error: Group is not viable.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>

	<span class="cm">/* Add the group to the container */</span>
	<span class="n">ret</span> <span class="o">=</span> <span class="n">ioctl</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">VFIO_GROUP_SET_CONTAINER</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">container</span><span class="p">);</span>
	<span class="k">if</span> <span class="p">(</span><span class="n">ret</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error: VFIO_GROUP_SET_CONTAINER.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>

	<span class="cm">/* Enable the IOMMU model we want */</span>
	<span class="n">ret</span> <span class="o">=</span> <span class="n">ioctl</span><span class="p">(</span><span class="n">container</span><span class="p">,</span> <span class="n">VFIO_SET_IOMMU</span><span class="p">,</span> <span class="n">VFIO_TYPE1_IOMMU</span><span class="p">);</span>
	<span class="k">if</span> <span class="p">(</span><span class="n">ret</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error: VFIO_SET_IOMMU.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>

	<span class="cm">/* Get addition IOMMU info */</span>
	<span class="n">ret</span> <span class="o">=</span> <span class="n">ioctl</span><span class="p">(</span><span class="n">container</span><span class="p">,</span> <span class="n">VFIO_IOMMU_GET_INFO</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">iommu_info</span><span class="p">);</span>
	<span class="k">if</span> <span class="p">(</span><span class="n">ret</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error: VFIO_IOMMU_GET_INFO.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>
	<span class="n">printf</span>
	    <span class="p">(</span><span class="s">"iommu_info.flags=0x%x, iommu_info.iova_pgsizes=0x%lx</span><span class="se">\n</span><span class="s">"</span><span class="p">,</span>
	     <span class="n">iommu_info</span><span class="p">.</span><span class="n">flags</span><span class="p">,</span> <span class="n">iommu_info</span><span class="p">.</span><span class="n">iova_pgsizes</span><span class="p">);</span>

	<span class="cm">/* Allocate some space and setup a DMA mapping */</span>
	<span class="n">dma_map</span><span class="p">.</span><span class="n">vaddr</span> <span class="o">=</span>
	    <span class="p">(</span><span class="kt">unsigned</span> <span class="kt">long</span> <span class="kt">long</span><span class="p">)</span><span class="n">mmap</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">,</span> <span class="n">PROT_READ</span> <span class="o">|</span> <span class="n">PROT_WRITE</span><span class="p">,</span>
				     <span class="n">MAP_PRIVATE</span> <span class="o">|</span> <span class="n">MAP_ANONYMOUS</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">);</span>
	<span class="n">dma_map</span><span class="p">.</span><span class="n">size</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">;</span>
	<span class="n">dma_map</span><span class="p">.</span><span class="n">iova</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>	<span class="cm">/* 1MB starting at 0x0 from device view */</span>
	<span class="n">dma_map</span><span class="p">.</span><span class="n">flags</span> <span class="o">=</span> <span class="n">VFIO_DMA_MAP_FLAG_READ</span> <span class="o">|</span> <span class="n">VFIO_DMA_MAP_FLAG_WRITE</span><span class="p">;</span>

	<span class="n">ret</span> <span class="o">=</span> <span class="n">ioctl</span><span class="p">(</span><span class="n">container</span><span class="p">,</span> <span class="n">VFIO_IOMMU_MAP_DMA</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">dma_map</span><span class="p">);</span>
	<span class="k">if</span> <span class="p">(</span><span class="n">ret</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error: VFIO_IOMMU_MAP_DMA.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>

	<span class="cm">/* Get a file descriptor for the device */</span>
	<span class="n">device</span> <span class="o">=</span> <span class="n">ioctl</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">VFIO_GROUP_GET_DEVICE_FD</span><span class="p">,</span> <span class="n">DEVICE_ID</span><span class="p">);</span>
	<span class="k">if</span> <span class="p">(</span><span class="n">device</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error: VFIO_GROUP_GET_DEVICE_FD.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>

	<span class="cm">/* Test and setup the device */</span>
	<span class="n">ret</span> <span class="o">=</span> <span class="n">ioctl</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">VFIO_DEVICE_GET_INFO</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">device_info</span><span class="p">);</span>
	<span class="k">if</span> <span class="p">(</span><span class="n">ret</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"Error: VFIO_DEVICE_GET_INFO.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
		<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
	<span class="p">}</span>
	<span class="n">printf</span>
	    <span class="p">(</span><span class="s">"device_info.flags=0x%x</span><span class="se">\t</span><span class="s">, device_info.num_regions=0x%x</span><span class="se">\t</span><span class="s">, device_info.num_irqs=0x%x</span><span class="se">\n</span><span class="s">"</span><span class="p">,</span>
	     <span class="n">device_info</span><span class="p">.</span><span class="n">flags</span><span class="p">,</span> <span class="n">device_info</span><span class="p">.</span><span class="n">num_regions</span><span class="p">,</span> <span class="n">device_info</span><span class="p">.</span><span class="n">num_irqs</span><span class="p">);</span>

	<span class="cm">/* index 0~5: bar space; can read/write/mmap
	 * index 6  : rom space; 
	 * index 7  : config space; can read/write */</span>
	<span class="k">for</span> <span class="p">(</span><span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span> <span class="n">i</span> <span class="o">&lt;</span> <span class="n">device_info</span><span class="p">.</span><span class="n">num_regions</span><span class="p">;</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span>
		<span class="k">struct</span> <span class="n">vfio_region_info</span> <span class="n">reg</span> <span class="o">=</span> <span class="p">{.</span><span class="n">argsz</span> <span class="o">=</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">reg</span><span class="p">)</span> <span class="p">};</span>

		<span class="n">reg</span><span class="p">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">i</span><span class="p">;</span>

		<span class="cm">/* Setup mappings... read/write offsets, mmaps
		 * For PCI devices, config space is a region */</span>
		<span class="n">ret</span> <span class="o">=</span> <span class="n">ioctl</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">VFIO_DEVICE_GET_REGION_INFO</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">reg</span><span class="p">);</span>
<span class="c">#if 0
		if (ret &lt; 0) {
			printf("Error: VFIO_DEVICE_GET_REGION_INFO.\n");
			return -1;
		}
#endif
</span>		<span class="n">printf</span>
		    <span class="p">(</span><span class="s">"index=%u, reg.flags=0x%x, cap_offset=0x%x, size=0x%lx</span><span class="se">\n</span><span class="s">"</span><span class="p">,</span>
		     <span class="n">reg</span><span class="p">.</span><span class="n">index</span><span class="p">,</span> <span class="n">reg</span><span class="p">.</span><span class="n">flags</span><span class="p">,</span> <span class="n">reg</span><span class="p">.</span><span class="n">cap_offset</span><span class="p">,</span> <span class="n">reg</span><span class="p">.</span><span class="n">size</span><span class="p">);</span>
	<span class="p">}</span>

	<span class="k">for</span> <span class="p">(</span><span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span> <span class="n">i</span> <span class="o">&lt;</span> <span class="n">device_info</span><span class="p">.</span><span class="n">num_irqs</span><span class="p">;</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span>
		<span class="k">struct</span> <span class="n">vfio_irq_info</span> <span class="n">irq</span> <span class="o">=</span> <span class="p">{.</span><span class="n">argsz</span> <span class="o">=</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">irq</span><span class="p">)</span> <span class="p">};</span>

		<span class="n">irq</span><span class="p">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">i</span><span class="p">;</span>

		<span class="n">ret</span> <span class="o">=</span> <span class="n">ioctl</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">VFIO_DEVICE_GET_IRQ_INFO</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">irq</span><span class="p">);</span>

		<span class="cm">/* Setup IRQs... eventfds, VFIO_DEVICE_SET_IRQS */</span>
		<span class="k">if</span> <span class="p">(</span><span class="n">ret</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span>
			<span class="n">printf</span><span class="p">(</span><span class="s">"Error: VFIO_DEVICE_GET_IRQ_INFO.</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
			<span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
		<span class="p">}</span>
		<span class="n">printf</span><span class="p">(</span><span class="s">"irq.flags=0x%x,irq.index=%u</span><span class="se">\t</span><span class="s">, irq.count=%u</span><span class="se">\n</span><span class="s">"</span><span class="p">,</span> <span class="n">irq</span><span class="p">.</span><span class="n">flags</span><span class="p">,</span> <span class="n">irq</span><span class="p">.</span><span class="n">index</span><span class="p">,</span> <span class="n">irq</span><span class="p">.</span><span class="n">count</span><span class="p">);</span>
	<span class="p">}</span>

	<span class="cm">/* Gratuitous device reset and go... */</span>
	<span class="n">ioctl</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">VFIO_DEVICE_RESET</span><span class="p">);</span>
<span class="p">}</span>


</pre></td></tr></tbody></table></code></div></div>

<h4 id="1devvfiovfio"><span class="me-2">1、/dev/vfio/vfio</span><a href="#1devvfiovfio" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>

<ol>
  <li>container</li>
</ol>

<div class="language-shell highlighter-rouge"><div class="code-header">
        <span data-label-text="Shell"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
</pre></td><td class="rouge-code"><pre>insmod vfio.ko
</pre></td></tr></tbody></table></code></div></div>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
</pre></td><td class="rouge-code"><pre><span class="k">static</span> <span class="k">struct</span> <span class="n">miscdevice</span> <span class="n">vfio_dev</span> <span class="o">=</span> <span class="p">{</span>
	<span class="p">.</span><span class="n">minor</span> <span class="o">=</span> <span class="n">VFIO_MINOR</span><span class="p">,</span>
	<span class="p">.</span><span class="n">name</span> <span class="o">=</span> <span class="s">"vfio"</span><span class="p">,</span>
	<span class="p">.</span><span class="n">fops</span> <span class="o">=</span> <span class="o">&amp;</span><span class="n">vfio_fops</span><span class="p">,</span>
	<span class="p">.</span><span class="n">nodename</span> <span class="o">=</span> <span class="s">"vfio/vfio"</span><span class="p">,</span>
	<span class="p">.</span><span class="n">mode</span> <span class="o">=</span> <span class="n">S_IRUGO</span> <span class="o">|</span> <span class="n">S_IWUGO</span><span class="p">,</span>
<span class="p">};</span>
</pre></td></tr></tbody></table></code></div></div>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
</pre></td><td class="rouge-code"><pre><span class="k">static</span> <span class="k">const</span> <span class="k">struct</span> <span class="n">file_operations</span> <span class="n">vfio_fops</span> <span class="o">=</span> <span class="p">{</span>
	<span class="p">.</span><span class="n">owner</span>		<span class="o">=</span> <span class="n">THIS_MODULE</span><span class="p">,</span>
	<span class="p">.</span><span class="n">open</span>		<span class="o">=</span> <span class="n">vfio_fops_open</span><span class="p">,</span>
	<span class="p">.</span><span class="n">release</span>	<span class="o">=</span> <span class="n">vfio_fops_release</span><span class="p">,</span>
	<span class="p">.</span><span class="n">read</span>		<span class="o">=</span> <span class="n">vfio_fops_read</span><span class="p">,</span>
	<span class="p">.</span><span class="n">write</span>		<span class="o">=</span> <span class="n">vfio_fops_write</span><span class="p">,</span>
	<span class="p">.</span><span class="n">unlocked_ioctl</span>	<span class="o">=</span> <span class="n">vfio_fops_unl_ioctl</span><span class="p">,</span>
	<span class="p">.</span><span class="n">compat_ioctl</span>	<span class="o">=</span> <span class="n">compat_ptr_ioctl</span><span class="p">,</span>
	<span class="p">.</span><span class="n">mmap</span>		<span class="o">=</span> <span class="n">vfio_fops_mmap</span><span class="p">,</span>
<span class="p">};</span>

<span class="p">.</span><span class="n">open</span><span class="o">:</span> <span class="k">struct</span> <span class="n">vfio_container</span> <span class="o">*</span><span class="n">container</span><span class="p">;</span> <span class="n">filep</span><span class="o">-&gt;</span><span class="n">private_data</span> <span class="o">=</span> <span class="n">container</span><span class="p">;</span> <span class="err">申请个</span><span class="n">container</span><span class="p">;</span>
<span class="p">.</span><span class="n">read</span><span class="o">:</span> <span class="n">container</span><span class="o">-&gt;</span><span class="n">iommu_driver</span><span class="o">-&gt;</span><span class="n">ops</span><span class="o">-&gt;</span><span class="n">read</span><span class="p">()</span> <span class="n">vfio_iommu_driver</span><span class="err">的</span><span class="n">read</span><span class="p">,</span> <span class="err">需要</span><span class="n">VFIO_SET_IOMMU</span><span class="err">之后才有效</span>
<span class="p">.</span><span class="n">write</span><span class="o">:</span> <span class="n">container</span><span class="o">-&gt;</span><span class="n">iommu_driver</span><span class="o">-&gt;</span><span class="n">ops</span><span class="o">-&gt;</span><span class="n">write</span><span class="p">()</span> <span class="n">vfio_iommu_driver</span><span class="err">的</span><span class="n">write</span>
<span class="p">.</span><span class="n">ioctl</span><span class="o">:</span> <span class="o">|</span> <span class="n">VFIO_GET_API_VERSION</span><span class="err">：</span> <span class="err">查看版本信息</span>
        <span class="o">|</span> <span class="n">VFIO_CHECK_EXTENSION</span><span class="err">：</span> <span class="n">check</span><span class="err">是否支持</span><span class="n">VFIO_TYPE1_IOMMU</span>
        <span class="o">|</span> <span class="n">VFIO_SET_IOMMU</span><span class="err">：</span> <span class="err">给</span><span class="n">container</span><span class="err">设置</span><span class="n">iommu</span><span class="err">操作</span>
        <span class="o">|</span> <span class="k">default</span><span class="err">：</span> <span class="n">driver</span><span class="o">-&gt;</span><span class="n">ops</span><span class="o">-&gt;</span><span class="n">ioctl</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">cmd</span><span class="p">,</span> <span class="n">arg</span><span class="p">);</span> <span class="err">转到</span><span class="n">IOMMU</span><span class="err">的</span><span class="n">cmd</span><span class="err">操作</span>
</pre></td></tr></tbody></table></code></div></div>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
</pre></td><td class="rouge-code"><pre><span class="n">list_for_each_entry</span><span class="p">(</span><span class="n">driver</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">vfio</span><span class="p">.</span><span class="n">iommu_drivers_list</span><span class="p">,</span> <span class="n">vfio_next</span><span class="p">)</span> <span class="p">{</span>
    <span class="k">if</span> <span class="p">(</span><span class="n">driver</span><span class="o">-&gt;</span><span class="n">ops</span><span class="o">-&gt;</span><span class="n">ioctl</span><span class="p">(</span><span class="nb">NULL</span><span class="p">,</span> <span class="n">VFIO_CHECK_EXTENSION</span><span class="p">,</span> <span class="n">arg</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span> <span class="c1">//从注册到的iommu_drivers_list中的所有iommu驱动中，找到支持arg的类型的iommu</span>
			<span class="n">module_put</span><span class="p">(</span><span class="n">driver</span><span class="o">-&gt;</span><span class="n">ops</span><span class="o">-&gt;</span><span class="n">owner</span><span class="p">);</span>
			<span class="k">continue</span><span class="p">;</span>
		<span class="p">}</span>

		<span class="n">data</span> <span class="o">=</span> <span class="n">driver</span><span class="o">-&gt;</span><span class="n">ops</span><span class="o">-&gt;</span><span class="n">open</span><span class="p">(</span><span class="n">arg</span><span class="p">);</span>
<span class="p">}</span>
</pre></td></tr></tbody></table></code></div></div>

<ol>
  <li>vfio iommu driver</li>
</ol>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
</pre></td><td class="rouge-code"><pre><span class="n">vfio_iommu_type1_init</span>
	<span class="o">-&gt;</span><span class="n">vfio_register_iommu_driver</span><span class="p">(</span><span class="o">&amp;</span><span class="p">(</span><span class="n">vfio_iommu_driver_ops_type1</span><span class="p">.</span><span class="n">ops</span><span class="p">));</span> <span class="c1">//为了VFIO_SET_IOMMU</span>
</pre></td></tr></tbody></table></code></div></div>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
</pre></td><td class="rouge-code"><pre><span class="k">static</span> <span class="k">const</span> <span class="k">struct</span> <span class="n">vfio_iommu_driver_compat_ops</span> <span class="n">vfio_iommu_driver_ops_type1</span> <span class="o">=</span> <span class="p">{</span>
	<span class="p">.</span><span class="n">ops</span> <span class="o">=</span> <span class="p">{</span>
		<span class="p">.</span><span class="n">name</span>			<span class="o">=</span> <span class="s">"vfio-iommu-type1"</span><span class="p">,</span>
		<span class="p">.</span><span class="n">owner</span>			<span class="o">=</span> <span class="n">THIS_MODULE</span><span class="p">,</span>
		<span class="p">.</span><span class="n">open</span>			<span class="o">=</span> <span class="n">vfio_iommu_type1_open</span><span class="p">,</span>
		<span class="p">.</span><span class="n">release</span>		<span class="o">=</span> <span class="n">vfio_iommu_type1_release</span><span class="p">,</span>
		<span class="p">.</span><span class="n">ioctl</span>			<span class="o">=</span> <span class="n">vfio_iommu_type1_ioctl</span><span class="p">,</span>
		<span class="p">.</span><span class="n">attach_group</span>		<span class="o">=</span> <span class="n">vfio_iommu_type1_attach_group</span><span class="p">,</span>
		<span class="p">.</span><span class="n">detach_group</span>		<span class="o">=</span> <span class="n">vfio_iommu_type1_detach_group</span><span class="p">,</span>
		<span class="p">.</span><span class="n">pin_pages</span>		<span class="o">=</span> <span class="n">vfio_iommu_type1_pin_pages</span><span class="p">,</span>
		<span class="p">.</span><span class="n">unpin_pages</span>		<span class="o">=</span> <span class="n">vfio_iommu_type1_unpin_pages</span><span class="p">,</span>
		<span class="p">.</span><span class="n">register_notifier</span>	<span class="o">=</span> <span class="n">vfio_iommu_type1_register_notifier</span><span class="p">,</span>
		<span class="p">.</span><span class="n">unregister_notifier</span>	<span class="o">=</span> <span class="n">vfio_iommu_type1_unregister_notifier</span><span class="p">,</span>
		<span class="p">.</span><span class="n">dma_rw</span>			<span class="o">=</span> <span class="n">vfio_iommu_type1_dma_rw</span><span class="p">,</span>
	<span class="p">},</span>
	<span class="p">.</span><span class="n">notify</span>				<span class="o">=</span> <span class="n">vfio_iommu_type1_notify</span><span class="p">,</span>
<span class="p">};</span>
<span class="n">ioctl</span><span class="o">:</span>
	<span class="k">switch</span> <span class="p">(</span><span class="n">cmd</span><span class="p">)</span> <span class="p">{</span> <span class="c1">//除了VFIO_CHECK_EXTENSION之外，都能对应上iommu的ops</span>
	<span class="k">case</span> <span class="n">VFIO_CHECK_EXTENSION</span><span class="p">:</span> <span class="c1">//支持哪些iommu driver type, 为了VFIO_SET_IOMMU来查看</span>
	<span class="k">case</span> <span class="n">VFIO_IOMMU_GET_INFO</span><span class="p">:</span> <span class="c1">//获取iommu的信息</span>
	<span class="k">case</span> <span class="n">VFIO_IOMMU_MAP_DMA</span><span class="p">:</span> <span class="c1">//dma映射</span>
	<span class="k">case</span> <span class="n">VFIO_IOMMU_UNMAP_DMA</span><span class="p">:</span><span class="c1">//dma解映射</span>
	<span class="k">case</span> <span class="n">VFIO_IOMMU_DIRTY_PAGES</span><span class="p">:</span><span class="c1">//支持iommu dirty</span>
	<span class="k">case</span> <span class="n">VFIO_IOMMU_BIND</span><span class="p">:</span> <span class="c1">//支持iommu sva的bind</span>
	<span class="k">case</span> <span class="n">VFIO_IOMMU_UNBIND</span><span class="p">:</span> <span class="c1">//支持iommu sva的unbind</span>
	<span class="k">case</span> <span class="n">VFIO_IOMMU_SET_PASID_TABLE</span><span class="p">:</span> <span class="c1">//attach a pasid table</span>
	<span class="k">case</span> <span class="n">VFIO_IOMMU_CACHE_INVALIDATE</span><span class="p">:</span><span class="c1">//invalidate translation caches</span>
	<span class="k">case</span> <span class="n">VFIO_IOMMU_SET_MSI_BINDING</span><span class="p">:</span> <span class="c1">//provides a stage1 giova/gpa MSI doorbell mapping</span>
	<span class="nl">default:</span>
		<span class="k">return</span> <span class="o">-</span><span class="n">ENOTTY</span><span class="p">;</span>
	<span class="p">}</span>
</pre></td></tr></tbody></table></code></div></div>

<h4 id="2devvfioiommu_group_id"><span class="me-2">2、/dev/vfio/$(iommu_group_id)</span><a href="#2devvfioiommu_group_id" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
9
10
11
12
13
14
</pre></td><td class="rouge-code"><pre><span class="c1">//vfio_pci.c</span>
<span class="k">static</span> <span class="k">const</span> <span class="k">struct</span> <span class="n">file_operations</span> <span class="n">vfio_group_fops</span> <span class="o">=</span> <span class="p">{</span>
	<span class="p">.</span><span class="n">owner</span>		<span class="o">=</span> <span class="n">THIS_MODULE</span><span class="p">,</span>
	<span class="p">.</span><span class="n">unlocked_ioctl</span>	<span class="o">=</span> <span class="n">vfio_group_fops_unl_ioctl</span><span class="p">,</span>
	<span class="p">.</span><span class="n">compat_ioctl</span>	<span class="o">=</span> <span class="n">compat_ptr_ioctl</span><span class="p">,</span>
	<span class="p">.</span><span class="n">open</span>		<span class="o">=</span> <span class="n">vfio_group_fops_open</span><span class="p">,</span>
	<span class="p">.</span><span class="n">release</span>	<span class="o">=</span> <span class="n">vfio_group_fops_release</span><span class="p">,</span>
<span class="p">};</span>

<span class="n">vfio_group_fops_unl_ioctl</span><span class="o">:</span>
	<span class="n">VFIO_GROUP_GET_STATUS</span><span class="o">:</span> <span class="c1">//获取group的状态，是否visable，有没有绑定container</span>
    <span class="n">VFIO_GROUP_SET_CONTAINER</span><span class="err">：</span> <span class="c1">//group绑定vfio container</span>
    <span class="n">VFIO_GROUP_UNSET_CONTAINER</span> <span class="c1">//group解绑定vfio container</span>
	<span class="n">VFIO_GROUP_GET_DEVICE_FD</span> <span class="c1">//根据device id获取struct vfio_device的fd，可以操作device</span>
</pre></td></tr></tbody></table></code></div></div>

<h4 id="3device-fd"><span class="me-2">3、device fd</span><a href="#3device-fd" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
</pre></td><td class="rouge-code"><pre><span class="n">vfio_pci_probe</span>
    <span class="o">-&gt;|</span> <span class="n">vfio</span> <span class="n">device</span><span class="o">-&gt;</span><span class="n">ops</span> <span class="o">=</span> <span class="n">ops</span><span class="p">;</span> <span class="c1">//vfio_pci_ops</span>
       <span class="n">vfio</span> <span class="n">device</span><span class="o">-&gt;</span><span class="n">dev</span> <span class="o">=</span> <span class="n">dev</span><span class="p">;</span> <span class="c1">//pci struct device</span>
	   <span class="n">vfio</span> <span class="n">device</span><span class="o">-&gt;</span><span class="n">drvdata</span> <span class="o">=</span> <span class="n">vdev</span><span class="p">;</span>
        <span class="n">pci</span> <span class="n">device</span><span class="o">-&gt;</span><span class="n">drvdata</span> <span class="o">=</span> <span class="n">vfio</span> <span class="n">device</span><span class="p">;</span>
</pre></td></tr></tbody></table></code></div></div>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
</pre></td><td class="rouge-code"><pre><span class="n">VFIO_GROUP_GET_DEVICE_FD</span> <span class="c1">//根据设备id match获得struct vfio_device</span>
    <span class="o">-&gt;|</span><span class="k">struct</span> <span class="n">vfio_device</span>
    <span class="o">-&gt;|</span><span class="n">ret</span> <span class="o">=</span> <span class="n">vfio_device</span><span class="o">-&gt;</span><span class="n">ops</span><span class="o">-&gt;</span><span class="n">open</span><span class="p">(</span><span class="n">device</span><span class="o">-&gt;</span><span class="n">device_data</span><span class="p">);</span> <span class="k">if</span> <span class="p">(</span><span class="n">ret</span><span class="p">)</span> <span class="k">return</span> <span class="n">ret</span>
    <span class="o">-&gt;|</span> <span class="k">if</span> <span class="n">device</span><span class="o">-&gt;</span><span class="n">ops</span><span class="o">-&gt;</span><span class="n">open</span><span class="err">失败了，分配个不使用的</span><span class="n">fd</span><span class="err">，创建</span><span class="n">anon_inode</span><span class="err">文件，挂上</span><span class="n">vfio_device_fops</span><span class="err">的</span><span class="n">ops</span><span class="p">,</span> <span class="k">return</span> <span class="n">fd</span><span class="p">.</span>	
</pre></td></tr></tbody></table></code></div></div>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
</pre></td><td class="rouge-code"><pre><span class="c1">//vfio_device_fops vs vfio_pci_ops</span>
    
<span class="n">vfio_device_fops</span> <span class="err">执行</span><span class="n">vfio_device</span><span class="o">-&gt;</span><span class="n">ops</span><span class="o">-&gt;</span><span class="n">xxx</span> <span class="o">=</span> <span class="n">vfio_pci_ops</span>
</pre></td></tr></tbody></table></code></div></div>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
</pre></td><td class="rouge-code"><pre><span class="n">vfio_pci_ops</span>
<span class="k">static</span> <span class="k">const</span> <span class="k">struct</span> <span class="n">vfio_device_ops</span> <span class="n">vfio_pci_ops</span> <span class="o">=</span> <span class="p">{</span>
	<span class="p">.</span><span class="n">name</span>		<span class="o">=</span> <span class="s">"vfio-pci"</span><span class="p">,</span>
	<span class="p">.</span><span class="n">open</span>		<span class="o">=</span> <span class="n">vfio_pci_open</span><span class="p">,</span>
	<span class="p">.</span><span class="n">release</span>	<span class="o">=</span> <span class="n">vfio_pci_release</span><span class="p">,</span>
	<span class="p">.</span><span class="n">ioctl</span>		<span class="o">=</span> <span class="n">vfio_pci_ioctl</span><span class="p">,</span>
	<span class="p">.</span><span class="n">read</span>		<span class="o">=</span> <span class="n">vfio_pci_read</span><span class="p">,</span>
	<span class="p">.</span><span class="n">write</span>		<span class="o">=</span> <span class="n">vfio_pci_write</span><span class="p">,</span>
	<span class="p">.</span><span class="n">mmap</span>		<span class="o">=</span> <span class="n">vfio_pci_mmap</span><span class="p">,</span>
	<span class="p">.</span><span class="n">request</span>	<span class="o">=</span> <span class="n">vfio_pci_request</span><span class="p">,</span>
	<span class="p">.</span><span class="n">match</span>		<span class="o">=</span> <span class="n">vfio_pci_match</span><span class="p">,</span>
<span class="p">};</span>

<span class="n">ioctl</span><span class="o">:</span>
    <span class="n">VFIO_DEVICE_GET_INFO</span><span class="o">:</span>
    <span class="n">VFIO_DEVICE_GET_REGION_INFO</span><span class="o">:</span> <span class="p">{</span>
        <span class="n">VFIO_PCI_CONFIG_REGION_INDEX</span>
        <span class="n">VFIO_PCI_BAR0_REGION_INDEX</span> <span class="p">...</span> <span class="n">VFIO_PCI_BAR5_REGION_INDEX</span>
        <span class="n">VFIO_PCI_ROM_REGION_INDEX</span>
        <span class="n">VFIO_PCI_VGA_REGION_INDEX</span>    
    <span class="p">}</span>
	<span class="n">VFIO_DEVICE_GET_IRQ_INFO</span><span class="o">:</span> <span class="p">{</span>
       	<span class="n">VFIO_PCI_INTX_IRQ_INDEX</span><span class="p">,</span>
        <span class="n">VFIO_PCI_MSI_IRQ_INDEX</span><span class="p">,</span>
        <span class="n">VFIO_PCI_MSIX_IRQ_INDEX</span><span class="p">,</span>
        <span class="n">VFIO_PCI_ERR_IRQ_INDEX</span><span class="p">,</span>
        <span class="n">VFIO_PCI_REQ_IRQ_INDEX</span><span class="p">,</span> 
    <span class="p">}</span>
	<span class="n">VFIO_DEVICE_SET_IRQS</span><span class="o">:</span>
	<span class="n">VFIO_DEVICE_RESET</span><span class="o">:</span>
	<span class="n">VFIO_DEVICE_GET_PCI_HOT_RESET_INFO</span><span class="o">:</span>
        
<span class="n">mmap</span><span class="o">:</span>
    
<span class="n">request</span><span class="o">:</span>
</pre></td></tr></tbody></table></code></div></div>

<h3 id="二设备透传实现"><span class="me-2">二、设备透传实现</span><a href="#二设备透传实现" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h3>

<p>在前面介绍VFIO的使用实例时，核心思想就是IOVA经过IOMMU映射出的物理地址与HVA经过MMU映射出的物理地址是同一个。对于设备透传的情况，先上图，然后看图说话。</p>

<p><a href="/posts/.\images\linux-vfio\设备透传.png" class="popup img-link "><img data-src="/posts/.\images\linux-vfio\设备透传.png" alt="img" class="lazyload" data-proofer-ignore></a></p>

<p><strong>这段话可以慢慢理解一下</strong></p>

<p>先来分析一下设备的DMA透传的工作流程，一旦设备透传给了虚机，虚机在配置设备DMA时直接使用GPA。此时GPA经由EPT会映射成HPA1，GPA经由IOMMU映射的地址为HPA2，此时的HPA1和HPA2必须相等，设备的透传才有意义。下面介绍在配置IOMMU时如何保证HPA1和HPA2相等，在VFIO章节讲到了VFIO_IOMMU_MAP_DMA这个命令就是将iova通过IOMMU映射到vaddr对应的物理地址上去。对于IOMMU来讲，此时的GPA就是iova，我们知道GPA经由EPT会映射为HPA1，对于VMM来讲，这个HPA1对应的虚机地址为HVA，那样的话在传入VFIO_IOMMU_MAP_DMA命令时讲hva作为vaddr，IOMMU就会将GPA映射为HVA对应的物理地址及HPA1，即HPA1和HPA2相等。上述流程帮助理清整个映射关系，实际映射IOMMU的操作很简单，前面提到了qemu维护了GPA和HVA的关系，在映射IOMMU的时候也可以派上用场。
注：<strong>IOMMU的映射在虚机启动时就已经建立好了，映射要涵盖整个GPA地址范围，同时虚机的HPA对应的物理页都不会交换出去（设备DMA交换是异步的）。</strong></p>

<h3 id="reference"><span class="me-2">reference:</span><a href="#reference" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h3>
<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
</pre></td><td class="rouge-code"><pre><span class="n">vfio</span><span class="err">概述</span><span class="p">(</span><span class="n">vfio</span><span class="o">/</span><span class="n">iommu</span><span class="o">/</span><span class="n">device</span> <span class="n">passthrough</span><span class="p">)</span>
<span class="n">https</span><span class="o">:</span><span class="c1">//www.cnblogs.com/yi-mu-xi/p/12370626.html</span>

<span class="n">vfio</span><span class="err">直通介绍</span>
<span class="n">https</span><span class="o">:</span><span class="c1">//luohao-brian.gitbooks.io/interrupt-virtualization/content/vfio-she-bei-zhi-tong-jian-jie.html</span>


<span class="n">VFIO</span> <span class="n">demo</span>
<span class="n">https</span><span class="o">:</span><span class="c1">//blog.csdn.net/qq_42138566/article/details/127573967</span>


<span class="n">VFIO</span><span class="err">概述</span>
<span class="n">https</span><span class="o">:</span><span class="c1">//blog.csdn.net/hx_op/article/details/104029622</span>

<span class="n">VFIO</span><span class="p">(</span><span class="n">Virtual</span> <span class="n">Function</span> <span class="n">IO</span><span class="p">)</span><span class="err">研究</span>
<span class="n">https</span><span class="o">:</span><span class="c1">//blog.csdn.net/21cnbao/article/details/115683410</span>

<span class="n">Linux</span><span class="err">内核：</span><span class="n">VFIO</span> <span class="err">内核文档</span> <span class="p">(</span><span class="err">实例，</span><span class="n">API</span><span class="err">，</span><span class="n">bus</span><span class="err">驱动</span><span class="n">API</span><span class="p">)</span>
<span class="n">https</span><span class="o">:</span><span class="c1">//rtoax.blog.csdn.net/article/details/110845720</span>

<span class="err">内核文档</span><span class="o">:</span> <span class="n">Documentation</span><span class="o">/</span><span class="n">driver</span><span class="o">-</span><span class="n">api</span><span class="o">/</span><span class="n">vfio</span><span class="p">.</span><span class="n">rst</span>
<span class="n">https</span><span class="o">:</span><span class="c1">//elixir.bootlin.com/linux/latest/source/Documentation/driver-api/vfio.rst</span>

<span class="n">MSI</span><span class="o">:</span> <span class="n">message</span> <span class="n">signal</span> <span class="n">interrupt</span>
</pre></td></tr></tbody></table></code></div></div>

</div>

<div class="post-tail-wrapper text-muted">

  <!-- categories -->
  
  <div class="post-meta mb-3">
    <i class="far fa-folder-open fa-fw me-1"></i>
    
      <a href='/categories/linux/'>linux</a>,
      <a href='/categories/vfio/'>VFIO</a>
  </div>
  
  <!-- post.html -->

  <!-- tags -->
  
  <div class="post-tags">
    <i class="fa fa-tags fa-fw me-1"></i>
      
      <a href="/tags/vfio/"
          class="post-tag no-text-decoration" >VFIO</a>
      
  </div>
  
  
  <div class="post-tail-bottom
    d-flex justify-content-between align-items-center mt-3 pt-5 pb-2">
    <div class="license-wrapper">

      

        

        This post is licensed under 
        <a href="https://creativecommons.org/licenses/by/4.0/">
          CC BY 4.0
        </a>
         by the author.

      
    </div>

    <!-- Post sharing snippet -->

<div class="share-wrapper">
  <span class="share-label text-muted me-1">Share</span>
  <span class="share-icons">
    
    
    

    
      
      <a
        href="https://twitter.com/intent/tweet?text=Kernel%20Virtual%20Function%20I/O%20-%20Cheng%20Luo&url=https%3A%2F%2Fluochenglcs.github.io%2Fposts%2Flinux-vfio%2F"
        data-bs-toggle="tooltip"
        data-bs-placement="top"
        title="Twitter"
        target="_blank"
        rel="noopener"
        aria-label="Twitter"
      >
        <i class="fa-fw fab fa-twitter"></i>
      </a>
    
      
      <a
        href="https://www.facebook.com/sharer/sharer.php?title=Kernel%20Virtual%20Function%20I/O%20-%20Cheng%20Luo&u=https%3A%2F%2Fluochenglcs.github.io%2Fposts%2Flinux-vfio%2F"
        data-bs-toggle="tooltip"
        data-bs-placement="top"
        title="Facebook"
        target="_blank"
        rel="noopener"
        aria-label="Facebook"
      >
        <i class="fa-fw fab fa-facebook-square"></i>
      </a>
    
      
      <a
        href="https://t.me/share/url?url=https%3A%2F%2Fluochenglcs.github.io%2Fposts%2Flinux-vfio%2F&text=Kernel%20Virtual%20Function%20I/O%20-%20Cheng%20Luo"
        data-bs-toggle="tooltip"
        data-bs-placement="top"
        title="Telegram"
        target="_blank"
        rel="noopener"
        aria-label="Telegram"
      >
        <i class="fa-fw fab fa-telegram"></i>
      </a>
    

    <i
      id="copy-link"
      class="fa-fw fas fa-link small"
      data-bs-toggle="tooltip"
      data-bs-placement="top"
      title="Copy link"
      data-title-succeed="Link copied successfully!"
    >
    </i>
  </span>
</div>


  </div><!-- .post-tail-bottom -->

</div><!-- div.post-tail-wrapper -->


      
    
      
    </div>
  </div>
  <!-- #core-wrapper -->

  <!-- panel -->
  <div id="panel-wrapper" class="col-xl-3 ps-2 text-muted">
    <div class="access">
      <!-- Get the last 5 posts from lastmod list. -->














  <div id="access-lastmod" class="post">
    <div class="panel-heading">Recently Updated</div>
    <ul class="post-content list-unstyled ps-0 pb-1 ms-1 mt-2">
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/%E5%86%85%E5%AD%98%E6%95%B0%E6%8D%AE%E5%BA%93/">MMDB (memory database)</a>
        </li>
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/userspace-rcu%E9%94%81/">userspace rcu lock</a>
        </li>
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/Real-time-operating-system/">Realtime Operating System</a>
        </li>
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/HUNDSUN-LightOS/">Hundsun Lightos</a>
        </li>
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/linux-struct-folio/">struct folio</a>
        </li>
      
    </ul>
  </div>
  <!-- #access-lastmod -->


      <!-- The trending tags list -->















  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        



  <div id="access-tags">
    <div class="panel-heading">Trending Tags</div>
    <div class="d-flex flex-wrap mt-3 mb-1 me-3">
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/lock/">lock</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/libumem/">libumem</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/mm/">mm</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/autonuma/">autonuma</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/bfs/">BFS</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/bus/">Bus</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/caladan/">caladan</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/damon/">damon</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/deep-learning/">Deep Learning</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/dma/">DMA</a>
      
    </div>
  </div>


    </div>

    
      
      



  <div id="toc-wrapper" class="ps-0 pe-4 mb-5">
    <div class="panel-heading ps-3 pt-2 mb-2">Contents</div>
    <nav id="toc"><ul><li>Kernel Virtual Function I/O<ul><li><a href="#一vfio">一、vfio</a><ul><li><a href="#1devvfiovfio">1、/dev/vfio/vfio</a></li><li><a href="#2devvfioiommu_group_id">2、/dev/vfio/$(iommu_group_id)</a></li><li><a href="#3device-fd">3、device fd</a></li></ul></li><li><a href="#二设备透传实现">二、设备透传实现</a></li><li><a href="#reference">reference:</a></li></ul></li></ul></li></ul></nav>
  </div>
  


    
  </div>
</div>

<!-- tail -->

  <div class="row">
    <div id="tail-wrapper" class="col-12 col-lg-11 col-xl-9 px-3 pe-xl-4 mt-5">
      
        
        <!--
  Recommend the other 3 posts according to the tags and categories of the current post,
  if the number is not enough, use the other latest posts to supplement.
-->

<!-- The total size of related posts -->


<!-- An random integer that bigger than 0 -->


<!-- Equals to TAG_SCORE / {max_categories_hierarchy} -->








  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  
    
  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  






<!-- Fill with the other newlest posts -->





  <div id="related-posts" class="mb-2 mb-sm-4">
    <h3 class="pt-2 mb-4 ms-1" data-toc-skip>
      Further Reading
    </h3>
    <div class="row row-cols-1 row-cols-md-2 row-cols-xl-3 g-4 mb-4">
      
        
        
        <div class="col">
          <a href="/posts/linux-kcsan/" class="card post-preview h-100">
            <div class="card-body">
              <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class="small"
  data-ts="1599782400"
  data-df="ll"
  
>
  Sep 11, 2020
</em>

              <h4 class="pt-0 my-2" data-toc-skip>The Kernel Concurrency Sanitizer (KCSAN)</h4>
              <div class="text-muted small">
                <p>
                  





                  1 kcasan

2 使用

                </p>
              </div>
            </div>
          </a>
        </div>
      
        
        
        <div class="col">
          <a href="/posts/linux-kprobe/" class="card post-preview h-100">
            <div class="card-body">
              <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class="small"
  data-ts="1600041600"
  data-df="ll"
  
>
  Sep 14, 2020
</em>

              <h4 class="pt-0 my-2" data-toc-skip>Kernel Probe (kprobe)</h4>
              <div class="text-muted small">
                <p>
                  





                  1 kprobe

2 debugfs kprobe trace
kprobe trace 使用debugfs的trace功能
 p[:[GRP/]EVENT] [MOD:]SYM[+offs]|MEMADDR [FETCHARGS]  : Set a probe
 r[MAXACTIVE][:[GRP/]EVENT] [MOD:]SYM[+0] [FETCHARGS]  : Set a r...
                </p>
              </div>
            </div>
          </a>
        </div>
      
        
        
        <div class="col">
          <a href="/posts/linux-userfaultfd/" class="card post-preview h-100">
            <div class="card-body">
              <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class="small"
  data-ts="1600905600"
  data-df="ll"
  
>
  Sep 24, 2020
</em>

              <h4 class="pt-0 my-2" data-toc-skip>Kernel userfaultfd</h4>
              <div class="text-muted small">
                <p>
                  





                  Userfault只支持匿名页，hugetlb、共享内存；

一、软件流程

1 初始化
调用__NR_userfaultfd syscall初始化
调用syscall初始化建立匿名inode文件，并初始化file-&amp;gt;private_data,并返回用户态文件fd。


  
    用户态：

    uffd = syscall(__NR_userfaultfd, O_CLOEXE...
                </p>
              </div>
            </div>
          </a>
        </div>
      
    </div>
    <!-- .card-deck -->
  </div>
  <!-- #related-posts -->


      
        
        <!-- Navigation buttons at the bottom of the post. -->

<div class="post-navigation d-flex justify-content-between">
  
    <a
      href="/posts/linux-slab/"
      class="btn btn-outline-primary"
      prompt="Older"
    >
      <p>linux Slab</p>
    </a>
  

  
    <a
      href="/posts/linux-damon/"
      class="btn btn-outline-primary"
      prompt="Newer"
    >
      <p>Kernel Data Access Monitor</p>
    </a>
  
</div>

      
        
        <!--  The comments switcher -->


      
    </div>
  </div>


        <!-- The Search results -->

<div id="search-result-wrapper" class="d-flex justify-content-center unloaded">
  <div class="col-11 post-content">
    <div id="search-hints">
      <!-- The trending tags list -->















  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        



  <div id="access-tags">
    <div class="panel-heading">Trending Tags</div>
    <div class="d-flex flex-wrap mt-3 mb-1 me-3">
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/lock/">lock</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/libumem/">libumem</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/mm/">mm</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/autonuma/">autonuma</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/bfs/">BFS</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/bus/">Bus</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/caladan/">caladan</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/damon/">damon</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/deep-learning/">Deep Learning</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/dma/">DMA</a>
      
    </div>
  </div>


    </div>
    <div id="search-results" class="d-flex flex-wrap justify-content-center text-muted mt-3"></div>
  </div>
</div>

      </div>
    </div>

    <!-- The Footer -->

<footer>
  <div class="container px-lg-4">
    <div class="d-flex justify-content-center align-items-center text-muted mx-md-3">
      <p>Using the <a href="https://jekyllrb.com" target="_blank" rel="noopener">Jekyll</a> theme <a href="https://github.com/cotes2020/jekyll-theme-chirpy" target="_blank" rel="noopener">Chirpy</a>
      </p>

      <p>©
        2023
        <a href="https://gitee.com/luochenglcs">luochunsheng</a>.
        
          <span
            data-bs-toggle="tooltip"
            data-bs-placement="top"
            title="Except where otherwise noted, the blog posts on this site are licensed under the Creative Commons Attribution 4.0 International (CC BY 4.0) License by the author."
          >Some rights reserved.</span>
        
      </p>
    </div>
  </div>
</footer>


    <div id="mask"></div>

    <button id="back-to-top" aria-label="back-to-top" class="btn btn-lg btn-box-shadow">
      <i class="fas fa-angle-up"></i>
    </button>

    
      <div
        id="notification"
        class="toast"
        role="alert"
        aria-live="assertive"
        aria-atomic="true"
        data-bs-animation="true"
        data-bs-autohide="false"
      >
        <div class="toast-header">
          <button
            type="button"
            class="btn-close ms-auto"
            data-bs-dismiss="toast"
            aria-label="Close"
          ></button>
        </div>
        <div class="toast-body text-center pt-0">
          <p class="px-2 mb-3">A new version of content is available.</p>
          <button type="button" class="btn btn-primary" aria-label="Update">
            Update
          </button>
        </div>
      </div>
    

    <!-- JS selector for site. -->

<!-- commons -->



<!-- layout specified -->


  

  
    <!-- image lazy-loading & popup & clipboard -->
    
  















  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  



  <script src="https://cdn.jsdelivr.net/combine/npm/jquery@3.7.0/dist/jquery.min.js,npm/bootstrap@5.2.3/dist/js/bootstrap.bundle.min.js,npm/simple-jekyll-search@1.10.0/dest/simple-jekyll-search.min.js,npm/lazysizes@5.3.2/lazysizes.min.js,npm/magnific-popup@1.1.0/dist/jquery.magnific-popup.min.js,npm/clipboard@2.0.11/dist/clipboard.min.js,npm/dayjs@1.11.7/dayjs.min.js,npm/dayjs@1.11.7/locale/en.min.js,npm/dayjs@1.11.7/plugin/relativeTime.min.js,npm/dayjs@1.11.7/plugin/localizedFormat.min.js,npm/tocbot@4.21.0/dist/tocbot.min.js"></script>






<script defer src="/assets/js/dist/post.min.js"></script>






    

    <!--
  Jekyll Simple Search loader
  See: <https://github.com/christian-fei/Simple-Jekyll-Search>
-->





<script>
  /* Note: dependent library will be loaded in `js-selector.html` */
  SimpleJekyllSearch({
    searchInput: document.getElementById('search-input'),
    resultsContainer: document.getElementById('search-results'),
    json: '/assets/js/data/search.json',
    searchResultTemplate: '<div class="px-1 px-sm-2 px-lg-4 px-xl-0">  <a href="{url}">{title}</a>  <div class="post-meta d-flex flex-column flex-sm-row text-muted mt-1 mb-1">    {categories}    {tags}  </div>  <p>{snippet}</p></div>',
    noResultsText: '<p class="mt-5"></p>',
    templateMiddleware: function(prop, value, template) {
      if (prop === 'categories') {
        if (value === '') {
          return `${value}`;
        } else {
          return `<div class="me-sm-4"><i class="far fa-folder fa-fw"></i>${value}</div>`;
        }
      }

      if (prop === 'tags') {
        if (value === '') {
          return `${value}`;
        } else {
          return `<div><i class="fa fa-tag fa-fw"></i>${value}</div>`;
        }
      }
    }
  });
</script>

  </body>
</html>

